package centroidtc;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class TextPaser {
	public int validNumOfText = 0;
	// store the subclass info
	public ArrayList<CategInfo> subClassItems = null;
	// store term's df in corpus
	private TreeMap<String, Integer> termDFDic = null;
	// store term's inter class df in corpus
	private TreeMap<String, Integer> termDFInterClass = null;
	// store term's inner class df in corpus
	private TreeMap<String, Integer> termDFInnerClass = null;
	// store a temp string
	private String aSubAndKeyWords = "";
	// store a stop words list
	private ArrayList<String> stopWordDic = null;
	// a temp tokenizer
	private Tokenizer tok = null;

	public TextPaser() {
		tok = new Tokenizer("");
		termDFDic = new TreeMap<String, Integer>();
		termDFInterClass = new TreeMap<String, Integer>();
		termDFInnerClass = new TreeMap<String, Integer>();
		
		setSubClassItems();
	}

	/**
	 * set sub class items info from a folder
	 */
	private void setSubClassItems() {

		subClassItems = new ArrayList<CategInfo>();
		File dir = new File("");
		String folderName = dir.getAbsolutePath() + File.separator + "20_newsgroups";

		String aBigStr = "";
		File path = new File(folderName);
		String[] tmp = null;
		if (path.isFile()) {
			System.out.println("input folder name should be a folder !");
			return;
		} else if (path.isDirectory()) {
			tmp = path.list();
			if (tmp.length > 0) {
				for (int i = 0; i < tmp.length; i++) {
					CategInfo cat = new CategInfo();
					File tf = new File(folderName + File.separator + tmp[i]);
					cat.setClassName(tmp[i]);
					if (tf.isFile()) {
						System.out.println("input folder should be a two-class folder!");
						return;
					} else if (tf.isDirectory()) {
						String[] ff = tf.list();
						if (ff.length > 0) {
							for (int j = 0; j < ff.length; j++) {
								File f = new File(ff[j]);
								aBigStr += f.getAbsoluteFile().getName() + ",";
							}
						}
					}
					cat.setDocId(aBigStr);
					this.subClassItems.add(cat);
					aBigStr = "";
				}
			}
		}
		System.out.println("finish in getting category and file information ...");
	}

	/**
	 * set corpus text info
	 */
	public void setCorpusItemsIntoFile() {
		// prepare a writer
		FileManager aWriter = new FileManager("20-newsgroup.txt");
		FileManager innerInfoWriter = new FileManager("20-inner_df.txt");
		ArrayList<String> tmpA = new ArrayList<String>();
		// to get stop words dictionary
		String stopWordFile = "conf" + File.separator + "stop";
		stopWordDic = this.configFile2Dic(stopWordFile);
		
		int catNum = 1;
		for (CategInfo cname : subClassItems) {
			ArrayList<String> fileList = parseLongString(cname.getDocId());
			for (String aFileName : fileList) {
				// get this file's text information
				ParsedItem pp = getTextInfo(cname.getClassName(), aFileName);
				// get this file's new name
				tmpA.add(catNum + "-" + aFileName);
				// parse text context
				String tt = this.parseTextItems(pp);
				if (!tt.equalsIgnoreCase("")) {
					tmpA.add(tt);
					// write a text's statistic information
					aWriter.writeArrayIntoFile(tmpA, "::");
					this.validNumOfText += 1;
					tmpA.clear();
				}
			}
			// write out inner-class distribution information
			ArrayList<String> ta = changeInnerMapIntoArray(termDFInnerClass);
			innerInfoWriter.writeArrayIntoFile(ta, ",");
			ta.clear();
			this.mergeInnerInfo(this.termDFInnerClass);
			this.termDFInnerClass.clear();
			System.out.println("finish in coping No-"
					+ catNum + " class: " + cname.getClassName());
			catNum++;
		}
		// write out the all IDF information
		writeOutMap(this.termDFDic,"20-all_idf.txt");
		// write out the inter-class distribution information
		writeOutMap(this.termDFInterClass,"20-inter_df.txt");
		this.termDFInterClass.clear();
		aWriter.closeWriter();
		innerInfoWriter.closeWriter();
		stopWordDic.clear();
	}

	/**
	 * parse a text info
	 */
	private String parseTextItems(ParsedItem p) {
		String ret = "";
		if (p == null) {
			return ret;
		}
		TreeMap<String, Integer> word2Feq = new TreeMap<String, Integer>();
		// parse text string
		Tokenizer t = new Tokenizer(p.getText());
		Integer count = 0;
		while (t.hasMoreElements()) {
			String aTmpWord = ((String) t.nextElement()).toLowerCase();
			aTmpWord = t.fixEasyWord(aTmpWord);
			if (t.isNumeric(aTmpWord) || stopWordDic.contains(aTmpWord)) {
				continue;
			}
			// filter text word with a length
			if ((aTmpWord != null) && (aTmpWord.trim().length() >= 3)) {
				count = (Integer) word2Feq.get(aTmpWord);
				// get the frequency of term
				word2Feq.put(aTmpWord, (count == null ? 1 : (count + 1)));
			}
		}
		// cope with the subject and key words
		Tokenizer tt = new Tokenizer(p.getTitle());
		this.aSubAndKeyWords = "";
		count = 0;
		while (tt.hasMoreElements()) {
			String aTmpWord = ((String) tt.nextElement()).toLowerCase();
			aTmpWord = tt.fixEasyWord(aTmpWord);
			if (tt.isNumeric(aTmpWord) || stopWordDic.contains(aTmpWord)) {
				continue;
			}
			// filter text word with a length
			if ((aTmpWord != null) && (aTmpWord.trim().length() >= 3)) {
				count = (Integer) word2Feq.get(aTmpWord);
				// get the frequency of term
				word2Feq.put(aTmpWord, (count == null ? 10 : (count + 10)));
			}
		}
		// ping a big string and set term's df info
		count = 0;
		for (Iterator<Map.Entry<String, Integer>> iter = word2Feq.entrySet()
				.iterator(); iter.hasNext();) {
			Map.Entry<String, Integer> pair = (Map.Entry<String, Integer>) iter
			.next();
			String aTerm = (String) pair.getKey();
			// threshold of the word frequency
			if ((pair.getValue() > 1)) {
				ret += aTerm + ":" + pair.getValue() + ",";
				// get the document frequency of term
				count = (Integer) this.termDFDic.get(aTerm);
				termDFDic.put(aTerm, (count == null ? 1 : (count + 1)));
				count = 0;
				// get the inner document frequency of term
				count = (Integer) this.termDFInnerClass.get(aTerm);
				termDFInnerClass.put(aTerm, (count == null ? 1 : (count + 1)));
			}
		}
		word2Feq.clear();
		return ret;
	}
	
	/**
	 * get the inner-class distribution information
	 */
	public ArrayList<String> changeInnerMapIntoArray(TreeMap<String,Integer> m){
		ArrayList<String> ret = new ArrayList<String>();
		for (Map.Entry<String, Integer> pair : m.entrySet()) {
			ret.add(pair.getKey() + ":" + pair.getValue());
		}
		return ret;
	}
	
	/**
	 * write out the all IDF information
	 */
	public void writeOutMap(TreeMap<String,Integer> m,String fileName){
		FileManager aWriter = new FileManager(fileName);
		for (Map.Entry<String, Integer> pair : m.entrySet()) {
			aWriter.writeBigStringIntoFile(pair.getKey() + ":" + pair.getValue());
		}
		aWriter.closeWriter();
		return;
	}
	
	/**
	 * merge a category's inner-class distribution information
	 */
	public void mergeInnerInfo(TreeMap<String,Integer> inner){
		Integer count = null;
		for (Map.Entry<String, Integer> pair : inner.entrySet()) {
			count = (Integer) this.termDFInterClass.get(pair.getKey());
			termDFInterClass.put(pair.getKey(), (count == null ? 1 : (count + 1)));
		}
		return;
	}
	
	/**
	 * parse a "word#" style's string info
	 */
	public ArrayList<String> parseLongString(String aBigStr) {
		ArrayList<String> ret = null;
		if (aBigStr == null) {
			return ret;
		}
		ret = new ArrayList<String>();
		String[] tmp = aBigStr.split(",");
		for (int i = 0; i < tmp.length; i++) {
			if (!tmp[i].equalsIgnoreCase("")) {
				ret.add(tmp[i]);
			}
		}
		return ret;
	}

	/**
	 * set up a dictionary from a config file
	 */
	public ArrayList<String> configFile2Dic(String aFileName) {
		ArrayList<String> ret = null;
		if (aFileName == null) {
			return ret;
		}

		ret = new ArrayList<String>();
		BufferedReader in = null;
		try {
			in = new BufferedReader(new FileReader(aFileName));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}

		String s = null;
		try {
			while ((s = in.readLine()) != null) {
				String ts = s.trim();
				if (!ret.contains(ts) && !ts.equalsIgnoreCase("")) {
					ret.add(ts);
				}
			}
			in.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return ret;
	}

	/**
	 * get useful string in a text file
	 */
	public ParsedItem getTextInfo(String className, String aFileName) {
		ParsedItem ret = null;
		if (aFileName == null || className == null) {
			return ret;
		}
		ret = new ParsedItem();
		File dir = new File("");
		String fileAdress = dir.getAbsolutePath() + File.separator + "20_newsgroups";
		fileAdress += File.separator + className + File.separator + aFileName;
		BufferedReader in = null;
		try {
			in = new BufferedReader(new FileReader(fileAdress));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		String tmpStr = "";
		String s = null;
		try {
			while ((s = in.readLine()) != null) {
				String ts = s.trim();
				// filter string with all kinds of filtering condition
				if (filterTextString(ts)) {
					tmpStr += tok.fixEasyWord(ts);
				}
			}
			in.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		ret.setId(aFileName);
		ret.setText(tmpStr);
		ret.setTitle(this.aSubAndKeyWords);
		this.aSubAndKeyWords = "";
		return ret;
	}

	/**
	 * get useful string in a text file
	 */
	public boolean filterTextString(String aInputStr) {
		boolean ret = false;
		// null ?
		if (aInputStr == null || aInputStr.equalsIgnoreCase("")) {
			return ret;
		}
		if (aInputStr.trim().length() < 2) {
			return ret;
		}

		Pattern p = null;
		Matcher m = null;
		ret = true;
		
		String aStr = aInputStr;
		
		// end with writes: ?
		p = Pattern.compile("(writes:)$");
		m = p.matcher(aStr);
		if (m.find()) {
			ret = false;
			return ret;
		}

		// email address ?
		p = Pattern.compile("(\\w)+.+@(.+.)+.+");
		m = p.matcher(aStr);
		if (m.find()) {
			ret = false;
			return ret;
		}
		// start with a sepcial word ?
		String specialWord = "(Newsgroups:)|(Path:)|(From:)|(Message-ID:)|";
		specialWord += "(Sender:)|(Organization:)|(References:)|(Date:)|";
		specialWord += "(Lines:)|(Distribution:)|(Reply-To:)|";
		specialWord += "(Xref:)|(Followup-To:)|(X-Newsreader:)";
		p = Pattern.compile("^(" + specialWord + ")");
		m = p.matcher(aStr);
		if (m.find()) {
			ret = false;
			return ret;
		}
		//cope with Keywords and subject
		p = Pattern.compile("^(Keywords:)");
		m = p.matcher(aStr);
		if (m.find()) {
			aSubAndKeyWords += m.replaceFirst("");
			ret = false;
			return ret;
		}
		
		//cope with Keywords and subject
		p = Pattern.compile("^(Subject:)");
		m = p.matcher(aStr);
		if (m.find()) {
			aSubAndKeyWords += m.replaceFirst("");
			ret = false;
			return ret;
		}
		// is a real word or term?
		p = Pattern.compile("(\\W){3}");
		String[] t = aStr.split(" ");
		for (int i = 0; i < t.length; i++) {
			m = p.matcher(t[i].trim());
			if (m.find()) {
				ret = false;
				return ret;
			} else if (t[i].length() > 20) {
				ret = false;
				return ret;
			}
		}
		return ret;
	}
	
	/**
	 * write out a text's word tfidf info
	 */
	public void writeTextTFIDFInfo(){
		// prepare a writer
		FileManager aWriter = new FileManager("20-newsgroup_vector.txt");
		String aFileName = "out" + File.separator + "20-newsgroup.txt";
		BufferedReader in = null;
		try {
			in = new BufferedReader(new FileReader(aFileName));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}

		String s = null;
		String[] ta = null;
		ArrayList<String> tmpA = new ArrayList<String>();
		try {
			while ((s = in.readLine()) != null) {
				String ts = s.trim();
				ta = ts.split("::");
				// add a file id in corpus
				tmpA.add(ta[0]);
				TreeMap<String,Integer> m = parseBigString(ta[1],",");
				String t = getTermTFIDFInfo(m);
				// add a vector information
				if (t != null) {
					tmpA.add(t);
					// write out this record
					aWriter.writeArrayIntoFile(tmpA, "::");
					m.clear();
				}
				tmpA.clear();
			}
			in.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		aWriter.closeWriter();
		return;
	}
	
	/**
	 * get a text's word tfidf info
	 */
	private String getTermTFIDFInfo(TreeMap<String,Integer> textMap) {
		String ret = null;
		TreeMap<String, Double> aTextTFIDFMap = null;
		if (textMap.size() <= 0) {
			return ret;
		}
		ret = "";
		// compute those tfidf's index
		Double tfidf = 0.0;
		Double sum = 0.0;
		int validNumOfText = this.validNumOfText;
		aTextTFIDFMap = new TreeMap<String, Double>();
		for (Map.Entry<String, Integer> pair : textMap.entrySet()) {
			double tf = pair.getValue() + 0.0;
			double idf = 0.0;
			Integer a_int = this.termDFDic.get((String) pair.getKey());
			idf = Math.log(validNumOfText / (a_int + 0.0));
			tfidf = tf * idf;
			aTextTFIDFMap.put((String) pair.getKey(), tfidf);
			sum += Math.pow(tfidf, 2);
			tfidf = 0.0;
		}

		Double base = Math.sqrt(sum);
		sum = 0.0;

		//String aBigWriteString = "";
		for (Iterator<Map.Entry<String, Double>> iter = aTextTFIDFMap
				.entrySet().iterator(); iter.hasNext();) {
			Map.Entry<String, Double> pair = (Map.Entry<String, Double>) iter
					.next();
			tfidf = (Double) pair.getValue() / base;
			ret += pair.getKey() + ":" + tfidf + ",";
			tfidf = 0.0;
		}
		aTextTFIDFMap.clear();
		return ret;
	}
	
	/**
	 * parse a "word:tfidf#" style's string info
	 */
	public TreeMap<String, Integer> parseBigString(String aBigStr,String token) {
		TreeMap<String, Integer> ret = null;
		if (aBigStr == null) {
			return ret;
		}
		ret = new TreeMap<String, Integer>();
		String[] tmp = aBigStr.split(token);
		for (int i = 0; i < tmp.length; i++) {
			if (!tmp[i].equalsIgnoreCase("")) {
				String[] s = tmp[i].split(":");
				if (s.length > 1) {
					ret.put(s[0], Integer.parseInt(s[1]));
				}
			}
		}
		return ret;
	}
	
	public static void main(String[] args) {
		TextPaser tm = new TextPaser();
		tm.setCorpusItemsIntoFile();
		tm.writeTextTFIDFInfo();
	}
}
